#!/usr/bin/python
# -*- coding: utf-8 -*-

import re
import urllib

def pro(m, n):
    cat[n] = unicode(m.group(2), 'cp936').encode('utf-8')
    id[n] = m.group(1)
    if n != 0:
    	father[n] = cat[n - 1]
    else:
    	father[n] = 'nobody~'
    print n, id[n], cat[n], father[n]
    cat_fh[n].write(cat[n] + '\n')
    id_fh[n].write(id[n] + '\n')
    father_fh[n].write(father[n] + '\n')

def page_count(s):
    m = re.search(r'</span><span style="font-size:14px;color:#666;">&nbsp;\(.*?(\d+).*?\)</span></div>', s)
    print m.group(1), '***'
    return (int(m.group(1)) + 9) / 10

cat = []
id = []
father = []

cat_fh = []
id_fh = []
father_fh = []

for i in range(0, 4):
	out = open('cat' + str(i), 'w')
	cat_fh.append(out)
	cat.append('')
	out = open('id' + str(i), 'w')
	id_fh.append(out)
	id.append('')
	out = open('father' + str(i), 'w')
	father_fh.append(out)
	father.append('')

reg = r'<h2><a href="http://baike\.baidu\.com/class/(\d+)\.html">(.*?)</a></h2>'
reg1 = r'<a name="\d+" id="(\d+)"></a>[\s\S]*?<div class="dirtit">(.*?)</div>([\S\s]*?)</table>'
reg2 = r'<td width="20%" class="f14"><a href="/taglist\?tag=(.*?)"target="_blank">(.*?)</a></td>'
reg3 = r'<font size="3"><a href="/view/(\d+)\.html\?fromTaglist" target="_blank">(.*?)</a></font><br />'

# crawl cats0
data = urllib.urlopen('http://baike.baidu.com/').read()
for match in re.finditer(reg, data):
    pro(match, 0)
    # crawl cats1
    data1 = urllib.urlopen('http://baike.baidu.com/class/' + id[0] + '.html').read()
    for match1 in re.finditer(reg1, data1):
        pro(match1, 1)
        # crawl cats2
        data2 = match1.group(3)
        for match2 in re.finditer(reg2, data2):
            pro(match2, 2)
            # crawl cats3
            data3 = urllib.urlopen('http://baike.baidu.com/taglist?tag=' + id[2]).read()
            pn = page_count(data3)
            for i in range(0, pn):
                cur_data = urllib.urlopen('http://baike.baidu.com/taglist?tag=' + id[2] + '&offset=' + str(i * 10)).read()
                print 'http://baike.baidu.com/taglist?tag=' + id[2] + '&offset=' + str(i * 10)
                for match3 in re.finditer(reg3, cur_data):
                    pro(match3, 3)

for i in range(0, 4):
    cat_fh[i].close()
    id_fh[i].close()
    father_fh[i].close()
